import requests
import time
import json



dlist = []


def getData(startDate="2020-01-20 00:00:00",endDate="2020-02-08 00:00:00"):
    url = "http://bjhhaxcsjjh.n.gongyibao.cn/api/trade/trade/jet/selectSimpleByPage"
    timeNow = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    params = {
        'pageNo':1,
        'pageSize':100,
        'pageState':1,
        'startDate':startDate,
        'endDate' :endDate
    }
    res = requests.get(url,params=params)
    res = json.loads(res.content.decode('utf-8'))
    count = res['dataPoints']['total']
    pageCount = res['dataPoints']['pages']
    print("总有"+str(count)+"条数据 共"+str(pageCount)+"页")
    for k in range(1,pageCount):
        print("正在处理第"+str(k)+"页...")
        params['pageNo']=k
        response = requests.get(url,params=params)
        response = json.loads(response.content.decode('utf-8'))
        list = response['dataPoints']['list']
        for item in list:
            ditem = {}
            ditem['jjh'] = "韩红爱心慈善基金会"
            ditem['jzdw'] = item['donationName']
            ditem['lx'] = '捐款'
            ditem['je'] = item['amount']
            ditem['rq'] = item['createTime'][0:10]
            ditem['jydh'] = item['tradeNo']
            dlist.append(ditem)


#完成调用爬虫及信息入库
from dg_cszg import models
import datetime

def run_spider():
    starttime = datetime.datetime.now()
    #getData("2020-01-21 00:00:00","2020-02-09 00:00:00")
    getData("2020-02-18 00:00:00", "2020-02-20 00:00:00")
    print(dlist)
    don_keys=["jjh" ,"jzdw" ,"rq" ,"lx" ,"rkwz", "slz" ,"jldw" ,"je" ,"jydh"]
    count=0
    fail=0
    for item in dlist:
        count = count + 1
        tmp_dict=item
        for tag in don_keys:
            if tag not in tmp_dict.keys():
                tmp_dict[tag]="null"
            if tmp_dict[tag] is None:
                print(tmp_dict,"||",count)
                tmp_dict[tag]="Error"
        tmp_text=tmp_dict["je"]
        tmp_text=str(tmp_text).replace(",","").replace("，","")
        tmp_dict["je"]=tmp_text
        try:
            record=models.donation_info (jjh=tmp_dict["jjh"], jzdw=tmp_dict["jzdw"], rq=tmp_dict["rq"], lx=tmp_dict["lx"],
                                     rkwz=tmp_dict["rkwz"], slz=tmp_dict["slz"], jldw=tmp_dict["jldw"],
                                     je =tmp_dict["je"],jydh=tmp_dict["jydh"])
            record.save()
        except:
            record=models.donation_info (jjh=tmp_dict["jjh"], jzdw="未知", rq=tmp_dict["rq"], lx=tmp_dict["lx"],
                                     rkwz=tmp_dict["rkwz"], slz=tmp_dict["slz"], jldw=tmp_dict["jldw"],
                                     je =tmp_dict["je"],jydh=tmp_dict["jydh"])
            record.save()
            fail=fail+1
            print("jzdw解析失败第",fail,"例子")
        print("正在入库韩红捐赠第",count,"例","还有",len(dlist)-count,"例")
    endtime = datetime.datetime.now()
    time=((endtime - starttime).seconds)/60
    print("爬取并入库韩红信息",count,"条,一共耗时",time,"min")
    return {"status":"ok"}
